{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2019-04-15 19:22:16--  http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz\n",
      "Resolving www.thespermwhale.com (www.thespermwhale.com)... 69.65.3.213\n",
      "Connecting to www.thespermwhale.com (www.thespermwhale.com)|69.65.3.213|:80... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 15719851 (15M) [application/x-gzip]\n",
      "Saving to: ‘tasks_1-20_v1-2.tar.gz’\n",
      "\n",
      "tasks_1-20_v1-2.tar 100%[===================>]  14.99M  9.70MB/s    in 1.5s    \n",
      "\n",
      "2019-04-15 19:22:18 (9.70 MB/s) - ‘tasks_1-20_v1-2.tar.gz’ saved [15719851/15719851]\n",
      "\n",
      "tasks_1-20_v1-2/\n",
      "tasks_1-20_v1-2/hn/\n",
      "tasks_1-20_v1-2/hn/qa16_basic-induction_train.txt\n",
      "tasks_1-20_v1-2/hn/qa13_compound-coreference_train.txt\n",
      "tasks_1-20_v1-2/hn/qa13_compound-coreference_test.txt\n",
      "tasks_1-20_v1-2/hn/qa14_time-reasoning_test.txt\n",
      "tasks_1-20_v1-2/hn/qa5_three-arg-relations_test.txt\n",
      "tasks_1-20_v1-2/hn/qa17_positional-reasoning_train.txt\n",
      "tasks_1-20_v1-2/hn/qa9_simple-negation_train.txt\n",
      "tasks_1-20_v1-2/hn/qa12_conjunction_train.txt\n",
      "tasks_1-20_v1-2/hn/qa6_yes-no-questions_train.txt\n",
      "tasks_1-20_v1-2/hn/qa2_two-supporting-facts_test.txt\n",
      "tasks_1-20_v1-2/hn/qa20_agents-motivations_train.txt\n",
      "tasks_1-20_v1-2/hn/qa7_counting_train.txt\n",
      "tasks_1-20_v1-2/hn/qa18_size-reasoning_test.txt\n",
      "tasks_1-20_v1-2/hn/qa1_single-supporting-fact_train.txt\n",
      "tasks_1-20_v1-2/hn/qa18_size-reasoning_train.txt\n",
      "tasks_1-20_v1-2/hn/qa1_single-supporting-fact_test.txt\n",
      "tasks_1-20_v1-2/hn/qa16_basic-induction_test.txt\n",
      "tasks_1-20_v1-2/hn/qa8_lists-sets_train.txt\n",
      "tasks_1-20_v1-2/hn/qa15_basic-deduction_test.txt\n",
      "tasks_1-20_v1-2/hn/qa11_basic-coreference_train.txt\n",
      "tasks_1-20_v1-2/hn/qa12_conjunction_test.txt\n",
      "tasks_1-20_v1-2/hn/qa10_indefinite-knowledge_test.txt\n",
      "tasks_1-20_v1-2/hn/qa19_path-finding_test.txt\n",
      "tasks_1-20_v1-2/hn/qa8_lists-sets_test.txt\n",
      "tasks_1-20_v1-2/hn/qa4_two-arg-relations_train.txt\n",
      "tasks_1-20_v1-2/hn/qa10_indefinite-knowledge_train.txt\n",
      "tasks_1-20_v1-2/hn/qa19_path-finding_train.txt\n",
      "tasks_1-20_v1-2/hn/qa20_agents-motivations_test.txt\n",
      "tasks_1-20_v1-2/hn/qa5_three-arg-relations_train.txt\n",
      "tasks_1-20_v1-2/hn/qa7_counting_test.txt\n",
      "tasks_1-20_v1-2/hn/qa3_three-supporting-facts_test.txt\n",
      "tasks_1-20_v1-2/hn/qa14_time-reasoning_train.txt\n",
      "tasks_1-20_v1-2/hn/qa17_positional-reasoning_test.txt\n",
      "tasks_1-20_v1-2/hn/qa9_simple-negation_test.txt\n",
      "tasks_1-20_v1-2/hn/qa4_two-arg-relations_test.txt\n",
      "tasks_1-20_v1-2/hn/qa6_yes-no-questions_test.txt\n",
      "tasks_1-20_v1-2/hn/qa15_basic-deduction_train.txt\n",
      "tasks_1-20_v1-2/hn/qa3_three-supporting-facts_train.txt\n",
      "tasks_1-20_v1-2/hn/qa2_two-supporting-facts_train.txt\n",
      "tasks_1-20_v1-2/hn/qa11_basic-coreference_test.txt\n",
      "tasks_1-20_v1-2/shuffled/\n",
      "tasks_1-20_v1-2/shuffled/qa16_basic-induction_train.txt\n",
      "tasks_1-20_v1-2/shuffled/qa13_compound-coreference_train.txt\n",
      "tasks_1-20_v1-2/shuffled/qa13_compound-coreference_test.txt\n",
      "tasks_1-20_v1-2/shuffled/qa14_time-reasoning_test.txt\n",
      "tasks_1-20_v1-2/shuffled/qa5_three-arg-relations_test.txt\n",
      "tasks_1-20_v1-2/shuffled/qa17_positional-reasoning_train.txt\n",
      "tasks_1-20_v1-2/shuffled/qa9_simple-negation_train.txt\n",
      "tasks_1-20_v1-2/shuffled/qa12_conjunction_train.txt\n",
      "tasks_1-20_v1-2/shuffled/qa6_yes-no-questions_train.txt\n",
      "tasks_1-20_v1-2/shuffled/qa2_two-supporting-facts_test.txt\n",
      "tasks_1-20_v1-2/shuffled/qa20_agents-motivations_train.txt\n",
      "tasks_1-20_v1-2/shuffled/qa7_counting_train.txt\n",
      "tasks_1-20_v1-2/shuffled/qa18_size-reasoning_test.txt\n",
      "tasks_1-20_v1-2/shuffled/qa1_single-supporting-fact_train.txt\n",
      "tasks_1-20_v1-2/shuffled/qa18_size-reasoning_train.txt\n",
      "tasks_1-20_v1-2/shuffled/qa1_single-supporting-fact_test.txt\n",
      "tasks_1-20_v1-2/shuffled/qa16_basic-induction_test.txt\n",
      "tasks_1-20_v1-2/shuffled/qa8_lists-sets_train.txt\n",
      "tasks_1-20_v1-2/shuffled/qa15_basic-deduction_test.txt\n",
      "tasks_1-20_v1-2/shuffled/qa11_basic-coreference_train.txt\n",
      "tasks_1-20_v1-2/shuffled/qa12_conjunction_test.txt\n",
      "tasks_1-20_v1-2/shuffled/qa10_indefinite-knowledge_test.txt\n",
      "tasks_1-20_v1-2/shuffled/qa19_path-finding_test.txt\n",
      "tasks_1-20_v1-2/shuffled/qa8_lists-sets_test.txt\n",
      "tasks_1-20_v1-2/shuffled/qa4_two-arg-relations_train.txt\n",
      "tasks_1-20_v1-2/shuffled/qa10_indefinite-knowledge_train.txt\n",
      "tasks_1-20_v1-2/shuffled/qa19_path-finding_train.txt\n",
      "tasks_1-20_v1-2/shuffled/qa20_agents-motivations_test.txt\n",
      "tasks_1-20_v1-2/shuffled/qa5_three-arg-relations_train.txt\n",
      "tasks_1-20_v1-2/shuffled/qa7_counting_test.txt\n",
      "tasks_1-20_v1-2/shuffled/qa3_three-supporting-facts_test.txt\n",
      "tasks_1-20_v1-2/shuffled/qa14_time-reasoning_train.txt\n",
      "tasks_1-20_v1-2/shuffled/qa17_positional-reasoning_test.txt\n",
      "tasks_1-20_v1-2/shuffled/qa9_simple-negation_test.txt\n",
      "tasks_1-20_v1-2/shuffled/qa4_two-arg-relations_test.txt\n",
      "tasks_1-20_v1-2/shuffled/qa6_yes-no-questions_test.txt\n",
      "tasks_1-20_v1-2/shuffled/qa15_basic-deduction_train.txt\n",
      "tasks_1-20_v1-2/shuffled/qa3_three-supporting-facts_train.txt\n",
      "tasks_1-20_v1-2/shuffled/qa2_two-supporting-facts_train.txt\n",
      "tasks_1-20_v1-2/shuffled/qa11_basic-coreference_test.txt\n",
      "tasks_1-20_v1-2/en-10k/\n",
      "tasks_1-20_v1-2/en-10k/qa16_basic-induction_train.txt\n",
      "tasks_1-20_v1-2/en-10k/qa13_compound-coreference_train.txt\n",
      "tasks_1-20_v1-2/en-10k/qa13_compound-coreference_test.txt\n",
      "tasks_1-20_v1-2/en-10k/qa14_time-reasoning_test.txt\n",
      "tasks_1-20_v1-2/en-10k/qa5_three-arg-relations_test.txt\n",
      "tasks_1-20_v1-2/en-10k/qa17_positional-reasoning_train.txt\n",
      "tasks_1-20_v1-2/en-10k/qa9_simple-negation_train.txt\n",
      "tasks_1-20_v1-2/en-10k/qa12_conjunction_train.txt\n",
      "tasks_1-20_v1-2/en-10k/qa6_yes-no-questions_train.txt\n",
      "tasks_1-20_v1-2/en-10k/qa2_two-supporting-facts_test.txt\n",
      "tasks_1-20_v1-2/en-10k/qa20_agents-motivations_train.txt\n",
      "tasks_1-20_v1-2/en-10k/qa7_counting_train.txt\n",
      "tasks_1-20_v1-2/en-10k/qa18_size-reasoning_test.txt\n",
      "tasks_1-20_v1-2/en-10k/qa1_single-supporting-fact_train.txt\n",
      "tasks_1-20_v1-2/en-10k/qa18_size-reasoning_train.txt\n",
      "tasks_1-20_v1-2/en-10k/qa1_single-supporting-fact_test.txt\n",
      "tasks_1-20_v1-2/en-10k/qa16_basic-induction_test.txt\n",
      "tasks_1-20_v1-2/en-10k/qa8_lists-sets_train.txt\n",
      "tasks_1-20_v1-2/en-10k/qa15_basic-deduction_test.txt\n",
      "tasks_1-20_v1-2/en-10k/qa11_basic-coreference_train.txt\n",
      "tasks_1-20_v1-2/en-10k/qa12_conjunction_test.txt\n",
      "tasks_1-20_v1-2/en-10k/qa10_indefinite-knowledge_test.txt\n",
      "tasks_1-20_v1-2/en-10k/qa19_path-finding_test.txt\n",
      "tasks_1-20_v1-2/en-10k/qa8_lists-sets_test.txt\n",
      "tasks_1-20_v1-2/en-10k/qa4_two-arg-relations_train.txt\n",
      "tasks_1-20_v1-2/en-10k/qa10_indefinite-knowledge_train.txt\n",
      "tasks_1-20_v1-2/en-10k/qa19_path-finding_train.txt\n",
      "tasks_1-20_v1-2/en-10k/qa20_agents-motivations_test.txt\n",
      "tasks_1-20_v1-2/en-10k/qa5_three-arg-relations_train.txt\n",
      "tasks_1-20_v1-2/en-10k/qa7_counting_test.txt\n",
      "tasks_1-20_v1-2/en-10k/qa3_three-supporting-facts_test.txt\n",
      "tasks_1-20_v1-2/en-10k/qa14_time-reasoning_train.txt\n",
      "tasks_1-20_v1-2/en-10k/qa17_positional-reasoning_test.txt\n",
      "tasks_1-20_v1-2/en-10k/qa9_simple-negation_test.txt\n",
      "tasks_1-20_v1-2/en-10k/qa4_two-arg-relations_test.txt\n",
      "tasks_1-20_v1-2/en-10k/qa6_yes-no-questions_test.txt\n",
      "tasks_1-20_v1-2/en-10k/qa15_basic-deduction_train.txt\n",
      "tasks_1-20_v1-2/en-10k/qa3_three-supporting-facts_train.txt\n",
      "tasks_1-20_v1-2/en-10k/qa2_two-supporting-facts_train.txt\n",
      "tasks_1-20_v1-2/en-10k/qa11_basic-coreference_test.txt\n",
      "tasks_1-20_v1-2/en-valid/\n",
      "tasks_1-20_v1-2/en-valid/qa8_test.txt\n",
      "tasks_1-20_v1-2/en-valid/qa11_train.txt\n",
      "tasks_1-20_v1-2/en-valid/qa7_train.txt\n",
      "tasks_1-20_v1-2/en-valid/qa13_train.txt\n",
      "tasks_1-20_v1-2/en-valid/qa7_valid.txt\n",
      "tasks_1-20_v1-2/en-valid/qa19_train.txt\n",
      "tasks_1-20_v1-2/en-valid/qa12_train.txt\n",
      "tasks_1-20_v1-2/en-valid/qa18_train.txt\n",
      "tasks_1-20_v1-2/en-valid/qa6_test.txt\n",
      "tasks_1-20_v1-2/en-valid/qa9_test.txt\n",
      "tasks_1-20_v1-2/en-valid/qa8_valid.txt\n",
      "tasks_1-20_v1-2/en-valid/qa2_train.txt\n",
      "tasks_1-20_v1-2/en-valid/qa12_test.txt\n",
      "tasks_1-20_v1-2/en-valid/qa11_test.txt\n",
      "tasks_1-20_v1-2/en-valid/qa9_train.txt\n",
      "tasks_1-20_v1-2/en-valid/qa1_test.txt\n",
      "tasks_1-20_v1-2/en-valid/qa7_test.txt\n",
      "tasks_1-20_v1-2/en-valid/qa16_train.txt\n",
      "tasks_1-20_v1-2/en-valid/qa4_train.txt\n",
      "tasks_1-20_v1-2/en-valid/qa2_valid.txt\n",
      "tasks_1-20_v1-2/en-valid/qa5_valid.txt\n",
      "tasks_1-20_v1-2/en-valid/qa16_test.txt\n",
      "tasks_1-20_v1-2/en-valid/qa18_valid.txt\n",
      "tasks_1-20_v1-2/en-valid/qa13_test.txt\n",
      "tasks_1-20_v1-2/en-valid/qa11_valid.txt\n",
      "tasks_1-20_v1-2/en-valid/qa1_valid.txt\n",
      "tasks_1-20_v1-2/en-valid/qa5_train.txt\n",
      "tasks_1-20_v1-2/en-valid/qa15_test.txt\n",
      "tasks_1-20_v1-2/en-valid/qa20_train.txt\n",
      "tasks_1-20_v1-2/en-valid/qa18_test.txt\n",
      "tasks_1-20_v1-2/en-valid/qa19_valid.txt\n",
      "tasks_1-20_v1-2/en-valid/qa9_valid.txt\n",
      "tasks_1-20_v1-2/en-valid/qa17_valid.txt\n",
      "tasks_1-20_v1-2/en-valid/qa15_valid.txt\n",
      "tasks_1-20_v1-2/en-valid/qa5_test.txt\n",
      "tasks_1-20_v1-2/en-valid/qa20_valid.txt\n",
      "tasks_1-20_v1-2/en-valid/qa14_valid.txt\n",
      "tasks_1-20_v1-2/en-valid/qa4_valid.txt\n",
      "tasks_1-20_v1-2/en-valid/qa15_train.txt\n",
      "tasks_1-20_v1-2/en-valid/qa10_test.txt\n",
      "tasks_1-20_v1-2/en-valid/qa8_train.txt\n",
      "tasks_1-20_v1-2/en-valid/qa6_train.txt\n",
      "tasks_1-20_v1-2/en-valid/qa17_train.txt\n",
      "tasks_1-20_v1-2/en-valid/qa10_train.txt\n",
      "tasks_1-20_v1-2/en-valid/qa3_test.txt\n",
      "tasks_1-20_v1-2/en-valid/qa3_train.txt\n",
      "tasks_1-20_v1-2/en-valid/qa16_valid.txt\n",
      "tasks_1-20_v1-2/en-valid/qa3_valid.txt\n",
      "tasks_1-20_v1-2/en-valid/qa14_test.txt\n",
      "tasks_1-20_v1-2/en-valid/qa19_test.txt\n",
      "tasks_1-20_v1-2/en-valid/qa4_test.txt\n",
      "tasks_1-20_v1-2/en-valid/qa1_train.txt\n",
      "tasks_1-20_v1-2/en-valid/qa2_test.txt\n",
      "tasks_1-20_v1-2/en-valid/qa13_valid.txt\n",
      "tasks_1-20_v1-2/en-valid/qa20_test.txt\n",
      "tasks_1-20_v1-2/en-valid/qa6_valid.txt\n",
      "tasks_1-20_v1-2/en-valid/qa14_train.txt\n",
      "tasks_1-20_v1-2/en-valid/qa12_valid.txt\n",
      "tasks_1-20_v1-2/en-valid/qa17_test.txt\n",
      "tasks_1-20_v1-2/en-valid/qa10_valid.txt\n",
      "tasks_1-20_v1-2/hn-10k/\n",
      "tasks_1-20_v1-2/hn-10k/qa16_basic-induction_train.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa13_compound-coreference_train.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa13_compound-coreference_test.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa14_time-reasoning_test.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa5_three-arg-relations_test.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa17_positional-reasoning_train.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa9_simple-negation_train.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa12_conjunction_train.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa6_yes-no-questions_train.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa2_two-supporting-facts_test.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa20_agents-motivations_train.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa7_counting_train.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa18_size-reasoning_test.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa1_single-supporting-fact_train.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa18_size-reasoning_train.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa1_single-supporting-fact_test.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa16_basic-induction_test.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa8_lists-sets_train.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa15_basic-deduction_test.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa11_basic-coreference_train.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa12_conjunction_test.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa10_indefinite-knowledge_test.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa19_path-finding_test.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa8_lists-sets_test.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa4_two-arg-relations_train.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa10_indefinite-knowledge_train.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa19_path-finding_train.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa20_agents-motivations_test.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa5_three-arg-relations_train.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa7_counting_test.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa3_three-supporting-facts_test.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa14_time-reasoning_train.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa17_positional-reasoning_test.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa9_simple-negation_test.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa4_two-arg-relations_test.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa6_yes-no-questions_test.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa15_basic-deduction_train.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa3_three-supporting-facts_train.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa2_two-supporting-facts_train.txt\n",
      "tasks_1-20_v1-2/hn-10k/qa11_basic-coreference_test.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/\n",
      "tasks_1-20_v1-2/shuffled-10k/qa16_basic-induction_train.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa13_compound-coreference_train.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa13_compound-coreference_test.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa14_time-reasoning_test.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa5_three-arg-relations_test.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa17_positional-reasoning_train.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa9_simple-negation_train.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa12_conjunction_train.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa6_yes-no-questions_train.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa2_two-supporting-facts_test.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa20_agents-motivations_train.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa7_counting_train.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa18_size-reasoning_test.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa1_single-supporting-fact_train.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa18_size-reasoning_train.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa1_single-supporting-fact_test.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa16_basic-induction_test.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa8_lists-sets_train.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa15_basic-deduction_test.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa11_basic-coreference_train.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa12_conjunction_test.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa10_indefinite-knowledge_test.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa19_path-finding_test.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa8_lists-sets_test.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa4_two-arg-relations_train.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa10_indefinite-knowledge_train.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa19_path-finding_train.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa20_agents-motivations_test.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa5_three-arg-relations_train.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa7_counting_test.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa3_three-supporting-facts_test.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa14_time-reasoning_train.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa17_positional-reasoning_test.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa9_simple-negation_test.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa4_two-arg-relations_test.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa6_yes-no-questions_test.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa15_basic-deduction_train.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa3_three-supporting-facts_train.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa2_two-supporting-facts_train.txt\n",
      "tasks_1-20_v1-2/shuffled-10k/qa11_basic-coreference_test.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/\n",
      "tasks_1-20_v1-2/en-valid-10k/qa8_test.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa11_train.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa7_train.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa13_train.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa7_valid.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa19_train.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa12_train.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa18_train.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa6_test.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa9_test.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa8_valid.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa2_train.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa12_test.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa11_test.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa9_train.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa1_test.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa7_test.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa16_train.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa4_train.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa2_valid.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa5_valid.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa16_test.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa18_valid.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa13_test.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa11_valid.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa1_valid.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa5_train.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa15_test.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa20_train.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa18_test.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa19_valid.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa9_valid.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa17_valid.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa15_valid.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa5_test.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa20_valid.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa14_valid.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa4_valid.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa15_train.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa10_test.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa8_train.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa6_train.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa17_train.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa10_train.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa3_test.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa3_train.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa16_valid.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa3_valid.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa14_test.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa19_test.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa4_test.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa1_train.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa2_test.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa13_valid.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa20_test.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa6_valid.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa14_train.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa12_valid.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa17_test.txt\n",
      "tasks_1-20_v1-2/en-valid-10k/qa10_valid.txt\n",
      "tasks_1-20_v1-2/en/\n",
      "tasks_1-20_v1-2/en/qa16_basic-induction_train.txt\n",
      "tasks_1-20_v1-2/en/qa13_compound-coreference_train.txt\n",
      "tasks_1-20_v1-2/en/qa13_compound-coreference_test.txt\n",
      "tasks_1-20_v1-2/en/qa14_time-reasoning_test.txt\n",
      "tasks_1-20_v1-2/en/qa5_three-arg-relations_test.txt\n",
      "tasks_1-20_v1-2/en/qa17_positional-reasoning_train.txt\n",
      "tasks_1-20_v1-2/en/qa9_simple-negation_train.txt\n",
      "tasks_1-20_v1-2/en/qa12_conjunction_train.txt\n",
      "tasks_1-20_v1-2/en/qa6_yes-no-questions_train.txt\n",
      "tasks_1-20_v1-2/en/qa2_two-supporting-facts_test.txt\n",
      "tasks_1-20_v1-2/en/qa20_agents-motivations_train.txt\n",
      "tasks_1-20_v1-2/en/qa7_counting_train.txt\n",
      "tasks_1-20_v1-2/en/qa18_size-reasoning_test.txt\n",
      "tasks_1-20_v1-2/en/qa1_single-supporting-fact_train.txt\n",
      "tasks_1-20_v1-2/en/qa18_size-reasoning_train.txt\n",
      "tasks_1-20_v1-2/en/qa1_single-supporting-fact_test.txt\n",
      "tasks_1-20_v1-2/en/qa16_basic-induction_test.txt\n",
      "tasks_1-20_v1-2/en/qa8_lists-sets_train.txt\n",
      "tasks_1-20_v1-2/en/qa15_basic-deduction_test.txt\n",
      "tasks_1-20_v1-2/en/qa11_basic-coreference_train.txt\n",
      "tasks_1-20_v1-2/en/qa12_conjunction_test.txt\n",
      "tasks_1-20_v1-2/en/qa10_indefinite-knowledge_test.txt\n",
      "tasks_1-20_v1-2/en/qa19_path-finding_test.txt\n",
      "tasks_1-20_v1-2/en/qa8_lists-sets_test.txt\n",
      "tasks_1-20_v1-2/en/qa4_two-arg-relations_train.txt\n",
      "tasks_1-20_v1-2/en/qa10_indefinite-knowledge_train.txt\n",
      "tasks_1-20_v1-2/en/qa19_path-finding_train.txt\n",
      "tasks_1-20_v1-2/en/qa20_agents-motivations_test.txt\n",
      "tasks_1-20_v1-2/en/qa5_three-arg-relations_train.txt\n",
      "tasks_1-20_v1-2/en/qa7_counting_test.txt\n",
      "tasks_1-20_v1-2/en/qa3_three-supporting-facts_test.txt\n",
      "tasks_1-20_v1-2/en/qa14_time-reasoning_train.txt\n",
      "tasks_1-20_v1-2/en/qa17_positional-reasoning_test.txt\n",
      "tasks_1-20_v1-2/en/qa9_simple-negation_test.txt\n",
      "tasks_1-20_v1-2/en/qa4_two-arg-relations_test.txt\n",
      "tasks_1-20_v1-2/en/qa6_yes-no-questions_test.txt\n",
      "tasks_1-20_v1-2/en/qa15_basic-deduction_train.txt\n",
      "tasks_1-20_v1-2/en/qa3_three-supporting-facts_train.txt\n",
      "tasks_1-20_v1-2/en/qa2_two-supporting-facts_train.txt\n",
      "tasks_1-20_v1-2/en/qa11_basic-coreference_test.txt\n",
      "tasks_1-20_v1-2/LICENSE.txt\n",
      "tasks_1-20_v1-2/README.txt\n"
     ]
    }
   ],
   "source": [
    "!wget http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz\n",
    "!tar -xvzf tasks_1-20_v1-2.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "keys = ['train', 'test']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _parse(file):\n",
    "    data, story = [], []\n",
    "    for line in open(file).readlines():\n",
    "        tid, text = line.rstrip('\\n').split(' ', 1)\n",
    "        if tid == '1':\n",
    "            story = []\n",
    "        # sentence\n",
    "        if text.endswith('.'):\n",
    "            story.append(text[:-1])\n",
    "        # question\n",
    "        else:\n",
    "            # remove any leading or trailing whitespace after splitting\n",
    "            query, answer, supporting = (x.strip() for x in text.split('\\t'))\n",
    "            substory = [x for x in story if x]\n",
    "            data.append((substory, query[:-1], answer))    # remove '?'\n",
    "            story.append(\"\")\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "tasks = ['qa1_single-supporting-fact_', 'qa2_two-supporting-facts_', 'qa3_three-supporting-facts_']\n",
    "data = {}\n",
    "for t in tasks :\n",
    "    data[t] = {}\n",
    "    for k in keys :\n",
    "        data[t][k] = list(zip(*_parse('tasks_1-20_v1-2/en-10k/' + t + k + '.txt')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "p, q, a = {}, {}, {}\n",
    "\n",
    "for k in keys :\n",
    "    p[k], q[k], a[k] = {}, {}, {}\n",
    "    for t in tasks :\n",
    "        p[k][t] = [\" . \".join(x) for x in data[t][k][0]]\n",
    "        q[k][t] = data[t][k][1]\n",
    "        a[k][t] = data[t][k][2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Transparency.preprocess.vectorizer import Vectorizer\n",
    "vec = Vectorizer(min_df=1)\n",
    "flatten = lambda x : [y for z in x for y in z]\n",
    "vec.fit(flatten(p['train'].values()) + flatten(q['train'].values()))\n",
    "\n",
    "entity_list = [k for k, v in vec.word2idx.items() if v >= 4]\n",
    "f = open('entity_list.txt', 'w')\n",
    "f.write(\"\\n\".join(entity_list))\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "for t in tasks :\n",
    "    df_paragraphs = []\n",
    "    df_questions = []\n",
    "    df_answers = []\n",
    "    df_exp_splits = []\n",
    "\n",
    "    for k in keys :\n",
    "        df_paragraphs += p[k][t]\n",
    "        df_questions += q[k][t]\n",
    "        df_answers += a[k][t]\n",
    "        df_exp_splits += [k] * len(p[k][t])\n",
    "\n",
    "    df = {'paragraph' : df_paragraphs, 'question' : df_questions, 'answer' : df_answers, 'exp_split' : df_exp_splits}\n",
    "    df = pd.DataFrame(df)\n",
    "    \n",
    "    from sklearn.model_selection import train_test_split\n",
    "    train_idx, dev_idx = train_test_split(df.index[df.exp_split == 'train'], test_size=0.15, random_state=16377)\n",
    "    df.loc[dev_idx, 'exp_split'] = 'dev'\n",
    "    df.to_csv('babi_' + t + 'dataset.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocabulary size :  24\n",
      "Vocabulary size :  38\n",
      "Vocabulary size :  39\n"
     ]
    }
   ],
   "source": [
    "for t in tasks :\n",
    "    dataset = 'babi_' + t + 'dataset.csv'\n",
    "    output_file = './vec_babi_' + t + '.p'\n",
    "    %run \"../preprocess_data_QA.py\" --data_file {dataset} --output_file {output_file} --all_answers_file entity_list.txt \\\n",
    "    --word_vectors_type None --min_df 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
